import java.util.Set;

import org.htmlparser.filters.LinkStringFilter;


public class DocCrawler {

	/**
	 * initial unvisited list using seeds
	 * @param seeds
	 */
	private void initCrawlerWithSeeds (String[] seeds) {
		for (int i = 0; i < seeds.length; i++) {
			LinkQueue.addUnvisitedUrl(seeds[i]);
		}
	}
	
	public void crawling (String[] seeds) {
		LinkStringFilter filter = new LinkStringFilter("http://lucene.apache.org/");
		//initial url queue
		initCrawlerWithSeeds(seeds);
		
		//while(!LinkQueue.unVisitedUrlsEmpty() && LinkQueue.getVisitedUrlNum() <= 1) {
			String visitUrl = (String) LinkQueue.unVisitedUrlDeQueue();
			//if (visitUrl == null) continue;
			DownLoadFile downLoader = new DownLoadFile();
			//download page
			downLoader.downloadFIle(visitUrl);
			LinkQueue.addVisitedUrl(visitUrl);
			Set<String> links = HtmlParserTool.extractLinks(visitUrl, filter);
			for (String link: links) {
				LinkQueue.addUnvisitedUrl(link);
			}
		//}
	}
}
